{
 "cells": [
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": [
    "%env LLM_BASE_URL=https://dashscope.aliyuncs.com/compatible-mode/v1\n",
    "%env LLM_API_KEY=sk-替换为自己的API Key\n",
    "\n",
    "%env JINA_SEGMENT_API_KEY=jina_替换为自己的API Key"
   ],
   "id": "cfd21a642e7a9e4a"
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "af375836-b870-458b-87d1-4e00565977eb",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-16T09:12:47.632961Z",
     "iopub.status.busy": "2024-12-16T09:12:47.632218Z",
     "iopub.status.idle": "2024-12-16T09:12:47.640998Z",
     "shell.execute_reply": "2024-12-16T09:12:47.638807Z",
     "shell.execute_reply.started": "2024-12-16T09:12:47.632890Z"
    },
    "papermill": {
     "duration": 0.115454,
     "end_time": "2024-11-23T14:29:00.919641",
     "exception": false,
     "start_time": "2024-11-23T14:29:00.804187",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%capture --no-stderr\n",
    "!pip install -U langchain langchain-community langchain-experimental langchain-openai pypdf sentence_transformers chromadb shutil"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "1e2c72b8-ee12-4130-af88-699998aa230c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-16T09:12:47.948434Z",
     "iopub.status.busy": "2024-12-16T09:12:47.947708Z",
     "iopub.status.idle": "2024-12-16T09:12:48.190357Z",
     "shell.execute_reply": "2024-12-16T09:12:48.189877Z",
     "shell.execute_reply.started": "2024-12-16T09:12:47.948367Z"
    },
    "papermill": {
     "duration": 0.319981,
     "end_time": "2024-11-23T14:29:01.380771",
     "exception": false,
     "start_time": "2024-11-23T14:29:01.060790",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "841d2b02-ad06-40d2-b11f-c7adccec6ca2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-16T09:12:48.264801Z",
     "iopub.status.busy": "2024-12-16T09:12:48.264569Z",
     "iopub.status.idle": "2024-12-16T09:12:48.429855Z",
     "shell.execute_reply": "2024-12-16T09:12:48.429211Z",
     "shell.execute_reply.started": "2024-12-16T09:12:48.264786Z"
    },
    "papermill": {
     "duration": 0.121409,
     "end_time": "2024-11-23T14:29:01.638126",
     "exception": false,
     "start_time": "2024-11-23T14:29:01.516717",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "expr_version = 'split_03_jiner_segment_api'\n",
    "\n",
    "preprocess_output_dir = os.path.join(os.path.pardir, 'outputs', 'v1_20240713')\n",
    "qa_df = pd.read_excel(os.path.join(preprocess_output_dir, 'question_answer.xlsx'))\n",
    "expr_dir = os.path.join(os.path.pardir, 'experiments', expr_version)\n",
    "\n",
    "os.makedirs(expr_dir, exist_ok=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cf7e81e3-4c82-4842-aef5-7592caaf1d39",
   "metadata": {
    "papermill": {
     "duration": 0.100379,
     "end_time": "2024-11-23T14:29:01.862379",
     "exception": false,
     "start_time": "2024-11-23T14:29:01.762000",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# 读取文档"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "e6920e29-bc7d-4635-be06-d151eaf0e100",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-16T09:13:08.817157Z",
     "iopub.status.busy": "2024-12-16T09:13:08.816819Z",
     "iopub.status.idle": "2024-12-16T09:13:10.473017Z",
     "shell.execute_reply": "2024-12-16T09:13:10.472544Z",
     "shell.execute_reply.started": "2024-12-16T09:13:08.817132Z"
    },
    "papermill": {
     "duration": 2.012298,
     "end_time": "2024-11-23T14:29:03.974974",
     "exception": false,
     "start_time": "2024-11-23T14:29:01.962676",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from langchain_community.document_loaders import PyPDFLoader\n",
    "import re\n",
    "\n",
    "loader = PyPDFLoader(os.path.join(os.path.pardir, 'data', '2024全球经济金融展望报告.pdf'))\n",
    "pdf_documents = loader.load()\n",
    "\n",
    "# 把页眉页脚去掉\n",
    "pattern = r\"^全球经济金融展望报告\\n中国银行研究院 \\d+ 2024年\"\n",
    "pdf_document = '\\n\\n'.join(re.sub(pattern, '', doc.page_content) for doc in pdf_documents)\n",
    "markdown_document = open(os.path.join(os.path.pardir, 'outputs', 'MinerU_parsed_20241204', '2024全球经济金融展望报告.md')).read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "575f15d5-8035-4451-9385-9acecac6bdf5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-16T09:13:17.775972Z",
     "iopub.status.busy": "2024-12-16T09:13:17.775756Z",
     "iopub.status.idle": "2024-12-16T09:13:17.780268Z",
     "shell.execute_reply": "2024-12-16T09:13:17.779863Z",
     "shell.execute_reply.started": "2024-12-16T09:13:17.775958Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "31202"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(pdf_document)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "b77b8fd4-38c3-4c8d-b913-d80926d6fb84",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-16T09:13:56.266121Z",
     "iopub.status.busy": "2024-12-16T09:13:56.265895Z",
     "iopub.status.idle": "2024-12-16T09:13:56.269808Z",
     "shell.execute_reply": "2024-12-16T09:13:56.269291Z",
     "shell.execute_reply.started": "2024-12-16T09:13:56.266102Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "33940"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(markdown_document)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "841ec659-4ad7-4e1f-b1ea-3477bf97fde3",
   "metadata": {
    "papermill": {
     "duration": 0.100297,
     "end_time": "2024-11-23T14:29:04.219302",
     "exception": false,
     "start_time": "2024-11-23T14:29:04.119005",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# 文档切分"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "74fe856a-7c19-4c3c-bb30-7abfa6298f74",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-16T09:21:42.929138Z",
     "iopub.status.busy": "2024-12-16T09:21:42.928483Z",
     "iopub.status.idle": "2024-12-16T09:21:42.938430Z",
     "shell.execute_reply": "2024-12-16T09:21:42.937448Z",
     "shell.execute_reply.started": "2024-12-16T09:21:42.929087Z"
    },
    "papermill": {
     "duration": 0.109229,
     "end_time": "2024-11-23T14:29:04.429069",
     "exception": false,
     "start_time": "2024-11-23T14:29:04.319840",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import pickle\n",
    "import requests\n",
    "from langchain.schema import Document\n",
    "\n",
    "def split_with_jina_api(text, max_len=700):\n",
    "    url = 'https://segment.jina.ai/'\n",
    "    headers = {\n",
    "        'Content-Type': 'application/json',\n",
    "        'Authorization': f\"Bearer {os.getenv('JINA_SEGMENT_API_KEY')}\"\n",
    "    }\n",
    "    data = {\n",
    "        \"content\": text,\n",
    "        \"return_tokens\": True,\n",
    "        \"return_chunks\": True,\n",
    "        \"max_chunk_length\": max_len\n",
    "    }\n",
    "    response = requests.post(url, headers=headers, json=data)\n",
    "    resp_json = response.json()\n",
    "    chunks = resp_json['chunks']\n",
    "    return [Document(page_content=chunk.strip()) for chunk in chunks if chunk.strip() != '']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "87e48987-95c4-499c-a1de-ba299d2e7709",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-16T09:22:09.650276Z",
     "iopub.status.busy": "2024-12-16T09:22:09.649505Z",
     "iopub.status.idle": "2024-12-16T09:22:12.492249Z",
     "shell.execute_reply": "2024-12-16T09:22:12.491773Z",
     "shell.execute_reply.started": "2024-12-16T09:22:09.650205Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "pdf_splitted_docs = split_with_jina_api(pdf_document)\n",
    "markdown_splitted_docs = split_with_jina_api(markdown_document)\n",
    "\n",
    "splitted_docs_dict = {\n",
    "    'pdf': pdf_splitted_docs,\n",
    "    'markdown': markdown_splitted_docs\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1652476c-3957-40fb-99ca-0a998390f094",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-13T03:00:55.978768Z",
     "iopub.status.busy": "2024-12-13T03:00:55.978064Z",
     "iopub.status.idle": "2024-12-13T03:00:55.987230Z",
     "shell.execute_reply": "2024-12-13T03:00:55.984859Z",
     "shell.execute_reply.started": "2024-12-13T03:00:55.978705Z"
    }
   },
   "source": [
    "## 检查一下切分后的块长度分布"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "2adff9b0-96c9-496d-8c19-20a7232c2f75",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-16T09:22:54.292432Z",
     "iopub.status.busy": "2024-12-16T09:22:54.292200Z",
     "iopub.status.idle": "2024-12-16T09:22:54.298832Z",
     "shell.execute_reply": "2024-12-16T09:22:54.298479Z",
     "shell.execute_reply.started": "2024-12-16T09:22:54.292413Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=============== pdf ===============\n",
      "docs count: 964\n",
      "doc length distribution\n",
      "count    964.000000\n",
      "mean      31.262448\n",
      "std       10.110942\n",
      "min        1.000000\n",
      "25%       28.000000\n",
      "50%       34.000000\n",
      "75%       37.000000\n",
      "90%       41.000000\n",
      "97%       44.110000\n",
      "99%       47.000000\n",
      "max       54.000000\n",
      "dtype: float64\n",
      "=============== markdown ===============\n",
      "docs count: 280\n",
      "doc length distribution\n",
      "count    280.000000\n",
      "mean     117.335714\n",
      "std      135.322745\n",
      "min        3.000000\n",
      "25%       20.000000\n",
      "50%       52.500000\n",
      "75%      182.750000\n",
      "90%      332.000000\n",
      "97%      451.930000\n",
      "99%      532.050000\n",
      "max      567.000000\n",
      "dtype: float64\n"
     ]
    }
   ],
   "source": [
    "for key, splitted_docs in splitted_docs_dict.items():\n",
    "    print(f\"=============== {key} ===============\")\n",
    "    print(f\"docs count: {len(splitted_docs)}\")\n",
    "    print('doc length distribution')\n",
    "    print(pd.Series([len(d.page_content) for d in splitted_docs]).describe([0.25, 0.5, 0.75, 0.9, 0.97, 0.99]))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "220dbc3a-fceb-4e49-a3f1-01e16660b2a6",
   "metadata": {
    "papermill": {
     "duration": 0.100209,
     "end_time": "2024-11-23T14:29:05.255871",
     "exception": false,
     "start_time": "2024-11-23T14:29:05.155662",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# 检索"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "2cddc9ac-88da-4c23-888d-481dc4a72f4a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-16T09:23:22.155905Z",
     "iopub.status.busy": "2024-12-16T09:23:22.155138Z",
     "iopub.status.idle": "2024-12-16T09:23:25.646968Z",
     "shell.execute_reply": "2024-12-16T09:23:25.646496Z",
     "shell.execute_reply.started": "2024-12-16T09:23:22.155835Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "device: cuda\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/anaconda3/lib/python3.10/site-packages/sentence_transformers/cross_encoder/CrossEncoder.py:11: TqdmExperimentalWarning: Using `tqdm.autonotebook.tqdm` in notebook mode. Use `tqdm.tqdm` instead to force console mode (e.g. in jupyter console)\n",
      "  from tqdm.autonotebook import tqdm, trange\n"
     ]
    }
   ],
   "source": [
    "from langchain.embeddings import HuggingFaceBgeEmbeddings\n",
    "from langchain_community.vectorstores import Chroma\n",
    "import torch\n",
    "\n",
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
    "print(f'device: {device}')\n",
    "\n",
    "def get_embeddings(model_path):\n",
    "    embeddings = HuggingFaceBgeEmbeddings(\n",
    "        model_name=model_path,\n",
    "        model_kwargs={'device': device},\n",
    "        encode_kwargs={'normalize_embeddings': True},\n",
    "        # show_progress=True\n",
    "        query_instruction='为这个句子生成表示以用于检索相关文章：'\n",
    "    )\n",
    "    return embeddings\n",
    "\n",
    "model_path = 'BAAI/bge-large-zh-v1.5'\n",
    "embeddings = get_embeddings(model_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "f6f46c73-7369-448f-a89a-ed3d817cad47",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-16T09:23:25.648023Z",
     "iopub.status.busy": "2024-12-16T09:23:25.647752Z",
     "iopub.status.idle": "2024-12-16T09:23:25.662530Z",
     "shell.execute_reply": "2024-12-16T09:23:25.662034Z",
     "shell.execute_reply.started": "2024-12-16T09:23:25.648010Z"
    },
    "papermill": {
     "duration": 83.983138,
     "end_time": "2024-11-23T14:35:06.117207",
     "exception": false,
     "start_time": "2024-11-23T14:33:42.134069",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import jieba\n",
    "import shutil\n",
    "\n",
    "from tqdm.auto import tqdm\n",
    "from langchain_community.vectorstores import Chroma\n",
    "\n",
    "def get_vector_db(splitted_docs, embeddings, name):\n",
    "    persist_directory = os.path.join(expr_dir, 'chroma', 'bge', name)\n",
    "    shutil.rmtree(persist_directory, ignore_errors=True)\n",
    "    vector_db = Chroma.from_documents(\n",
    "        splitted_docs,\n",
    "        embedding=embeddings,\n",
    "        persist_directory=persist_directory\n",
    "    )\n",
    "    return vector_db"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "11e148bd-7a3f-4bde-a574-21a30f337542",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-16T09:23:30.425489Z",
     "iopub.status.busy": "2024-12-16T09:23:30.424713Z",
     "iopub.status.idle": "2024-12-16T09:24:08.647172Z",
     "shell.execute_reply": "2024-12-16T09:24:08.646620Z",
     "shell.execute_reply.started": "2024-12-16T09:23:30.425422Z"
    }
   },
   "outputs": [],
   "source": [
    "vector_db_dict = {\n",
    "    key: get_vector_db(splitted_docs, embeddings, f\"{key}_chroma\")\n",
    "    for key, splitted_docs in splitted_docs_dict.items()\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7925564a-7d30-4914-baaf-4a00abb7686d",
   "metadata": {
    "papermill": {
     "duration": 0.109216,
     "end_time": "2024-11-23T14:35:26.464009",
     "exception": false,
     "start_time": "2024-11-23T14:35:26.354793",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# 生成答案"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "27132c3b-0051-4df6-bf57-fd804acb8d17",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-16T09:24:30.515795Z",
     "iopub.status.busy": "2024-12-16T09:24:30.515622Z",
     "iopub.status.idle": "2024-12-16T09:24:30.582095Z",
     "shell.execute_reply": "2024-12-16T09:24:30.581618Z",
     "shell.execute_reply.started": "2024-12-16T09:24:30.515782Z"
    },
    "papermill": {
     "duration": 0.199165,
     "end_time": "2024-11-23T14:35:27.323500",
     "exception": false,
     "start_time": "2024-11-23T14:35:27.124335",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_3287872/3342461511.py:3: LangChainDeprecationWarning: The class `Ollama` was deprecated in LangChain 0.3.1 and will be removed in 1.0.0. An updated version of the class exists in the :class:`~langchain-ollama package and should be used instead. To use it run `pip install -U :class:`~langchain-ollama` and import as `from :class:`~langchain_ollama import OllamaLLM``.\n",
      "  ollama_llm = Ollama(\n"
     ]
    }
   ],
   "source": [
    "from langchain.llms import Ollama\n",
    "\n",
    "ollama_llm = Ollama(\n",
    "    model='qwen2:7b-instruct',\n",
    "    base_url='http://localhost:11434',\n",
    "    top_k=1\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "50404beb-3be0-4aaa-b124-8c7a52b84531",
   "metadata": {
    "editable": true,
    "execution": {
     "iopub.execute_input": "2024-12-16T09:24:31.339251Z",
     "iopub.status.busy": "2024-12-16T09:24:31.338798Z",
     "iopub.status.idle": "2024-12-16T09:24:31.349943Z",
     "shell.execute_reply": "2024-12-16T09:24:31.347935Z",
     "shell.execute_reply.started": "2024-12-16T09:24:31.339214Z"
    },
    "papermill": {
     "duration": 0.159318,
     "end_time": "2024-11-23T14:35:26.768506",
     "exception": false,
     "start_time": "2024-11-23T14:35:26.609188",
     "status": "completed"
    },
    "slideshow": {
     "slide_type": ""
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import time\n",
    "\n",
    "def rag(vector_db, llm, query, n_chunks=4):\n",
    "    prompt_tmpl = \"\"\"\n",
    "你是一个金融分析师，擅长根据所获取的信息片段，对问题进行分析和推理。\n",
    "你的任务是根据所获取的信息片段（<<<<context>>><<<</context>>>之间的内容）回答问题。\n",
    "回答保持简洁，不必重复问题，不要添加描述性解释和与答案无关的任何内容。\n",
    "已知信息：\n",
    "<<<<context>>>\n",
    "{{knowledge}}\n",
    "<<<</context>>>\n",
    "\n",
    "问题：{{query}}\n",
    "请回答：\n",
    "\"\"\".strip()\n",
    "    chunks = vector_db.similarity_search(query, k=n_chunks)\n",
    "    prompt = prompt_tmpl.replace('{{knowledge}}', '\\n\\n'.join([doc.page_content for doc in chunks])).replace('{{query}}', query)\n",
    "    retry_count = 3\n",
    "\n",
    "    resp = ''\n",
    "    while retry_count > 0:\n",
    "        try:\n",
    "            resp = llm.invoke(prompt)\n",
    "            break\n",
    "        except Exception as e:\n",
    "            retry_count -= 1\n",
    "            sleeping_seconds = 2 ** (4 - retry_count)\n",
    "            print(f\"query={query}, error={e}, sleeping={sleeping_seconds}, remaining retry count={retry_count}\")\n",
    "            \n",
    "            time.sleep(sleeping_seconds)\n",
    "    \n",
    "    return resp, chunks"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "95e5a804-2dc6-411c-ba71-6ccf765b2b73",
   "metadata": {
    "papermill": {
     "duration": 0.135973,
     "end_time": "2024-11-23T14:35:27.001401",
     "exception": false,
     "start_time": "2024-11-23T14:35:26.865428",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## 预测"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "166392d8-f801-4372-b8ad-3e79aef0b350",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-16T09:24:33.293060Z",
     "iopub.status.busy": "2024-12-16T09:24:33.292881Z",
     "iopub.status.idle": "2024-12-16T09:24:33.298267Z",
     "shell.execute_reply": "2024-12-16T09:24:33.297842Z",
     "shell.execute_reply.started": "2024-12-16T09:24:33.293046Z"
    },
    "papermill": {
     "duration": 0.141864,
     "end_time": "2024-11-23T14:35:27.564409",
     "exception": false,
     "start_time": "2024-11-23T14:35:27.422545",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "prediction_df = qa_df[qa_df['dataset'] == 'test'][['uuid', 'question', 'qa_type', 'answer']].rename(columns={'answer': 'ref_answer'})\n",
    "\n",
    "def predict(vector_db, llm, prediction_df, n_chunks):\n",
    "    prediction_df = prediction_df.copy()\n",
    "    answer_dict = {}\n",
    "\n",
    "    for idx, row in tqdm(prediction_df.iterrows(), total=len(prediction_df)):\n",
    "        uuid = row['uuid']\n",
    "        question = row['question']\n",
    "        answer, chunks = rag(vector_db, llm, question, n_chunks=n_chunks)\n",
    "        assert len(chunks) <= n_chunks\n",
    "        answer_dict[question] = {\n",
    "            'uuid': uuid,\n",
    "            'ref_answer': row['ref_answer'],\n",
    "            'gen_answer': answer,\n",
    "            'chunks': chunks\n",
    "        }\n",
    "\n",
    "    prediction_df.loc[:, 'gen_answer'] = prediction_df['question'].apply(lambda q: answer_dict[q]['gen_answer'])\n",
    "    prediction_df.loc[:, 'chunks'] = prediction_df['question'].apply(lambda q: answer_dict[q]['chunks'])\n",
    "\n",
    "    return prediction_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "f881aa9c-bf51-4120-95ca-9108b53e48cb",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-16T09:24:34.421067Z",
     "iopub.status.busy": "2024-12-16T09:24:34.420281Z",
     "iopub.status.idle": "2024-12-16T09:24:34.430293Z",
     "shell.execute_reply": "2024-12-16T09:24:34.427933Z",
     "shell.execute_reply.started": "2024-12-16T09:24:34.420996Z"
    }
   },
   "outputs": [],
   "source": [
    "save_path = os.path.join(expr_dir, 'preds.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "f40b1241-fb9a-43dd-a755-bf9409e90be2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-16T09:24:35.054519Z",
     "iopub.status.busy": "2024-12-16T09:24:35.054258Z",
     "iopub.status.idle": "2024-12-16T09:24:35.058318Z",
     "shell.execute_reply": "2024-12-16T09:24:35.057710Z",
     "shell.execute_reply.started": "2024-12-16T09:24:35.054498Z"
    }
   },
   "outputs": [],
   "source": [
    "if os.path.exists(save_path):\n",
    "    print(f'found cache at {save_path}')\n",
    "    pred_dict = pickle.load(open(save_path, 'rb'))\n",
    "else:\n",
    "    pred_dict = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "5b3dff59-03e8-4343-b12f-66c16b4bfa33",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-16T09:24:37.608496Z",
     "iopub.status.busy": "2024-12-16T09:24:37.608319Z",
     "iopub.status.idle": "2024-12-16T09:27:28.990757Z",
     "shell.execute_reply": "2024-12-16T09:27:28.990269Z",
     "shell.execute_reply.started": "2024-12-16T09:24:37.608482Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "prediting for pdf ...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d740839f257440e8a3b65669fadc4805",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "prediting for markdown ...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "43b82f93c2904439b2045cae77c33280",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "for key, vector_db in vector_db_dict.items():\n",
    "    print(f'prediting for {key} ...')\n",
    "    if key in pred_dict:\n",
    "        continue\n",
    "    pred_df = predict(vector_db, ollama_llm, prediction_df, n_chunks=3)\n",
    "    pred_dict[key] = pred_df\n",
    "    pickle.dump(pred_dict, open(save_path, 'wb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7d79e974-089f-4c08-ba5e-804f6542e06a",
   "metadata": {
    "papermill": {
     "duration": 0.14423,
     "end_time": "2024-11-23T14:44:03.513124",
     "exception": false,
     "start_time": "2024-11-23T14:44:03.368894",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# 评估"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "217568fe-c0e4-49eb-9a7c-9fdfbc033d8a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-16T09:29:09.315870Z",
     "iopub.status.busy": "2024-12-16T09:29:09.315635Z",
     "iopub.status.idle": "2024-12-16T09:29:09.520406Z",
     "shell.execute_reply": "2024-12-16T09:29:09.519898Z",
     "shell.execute_reply.started": "2024-12-16T09:29:09.315851Z"
    },
    "papermill": {
     "duration": 0.369729,
     "end_time": "2024-11-23T14:44:04.017198",
     "exception": false,
     "start_time": "2024-11-23T14:44:03.647469",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from langchain_openai import ChatOpenAI\n",
    "import time\n",
    "\n",
    "judge_llm = ChatOpenAI(\n",
    "    api_key=os.environ['LLM_API_KEY'],\n",
    "    base_url=os.environ['LLM_BASE_URL'],\n",
    "    model_name='qwen2-72b-instruct',\n",
    "    temperature=0\n",
    ")\n",
    "\n",
    "def evaluate(prediction_df):\n",
    "    \"\"\"\n",
    "    对预测结果进行打分\n",
    "    :param prediction_df: 预测结果，需要包含问题，参考答案，生成的答案，列名分别为question, ref_answer, gen_answer\n",
    "    :return 打分模型原始返回结果\n",
    "    \"\"\"\n",
    "    prompt_tmpl = \"\"\"\n",
    "你是一个经济学博士，现在我有一系列问题，有一个助手已经对这些问题进行了回答，你需要参照参考答案，评价这个助手的回答是否正确，仅回复“是”或“否”即可，不要带其他描述性内容或无关信息。\n",
    "问题：\n",
    "<question>\n",
    "{{question}}\n",
    "</question>\n",
    "\n",
    "参考答案：\n",
    "<ref_answer>\n",
    "{{ref_answer}}\n",
    "</ref_answer>\n",
    "\n",
    "助手回答：\n",
    "<gen_answer>\n",
    "{{gen_answer}}\n",
    "</gen_answer>\n",
    "请评价：\n",
    "    \"\"\"\n",
    "    results = []\n",
    "\n",
    "    for _, row in tqdm(prediction_df.iterrows(), total=len(prediction_df)):\n",
    "        question = row['question']\n",
    "        ref_answer = row['ref_answer']\n",
    "        gen_answer = row['gen_answer']\n",
    "\n",
    "        prompt = prompt_tmpl.replace('{{question}}', question).replace('{{ref_answer}}', str(ref_answer)).replace('{{gen_answer}}', gen_answer).strip()\n",
    "        \n",
    "        retry_count = 3\n",
    "        result = ''\n",
    "        \n",
    "        while retry_count > 0:\n",
    "            try:\n",
    "                result = judge_llm.invoke(prompt).content\n",
    "                break\n",
    "            except Exception as e:\n",
    "                retry_count -= 1\n",
    "                sleeping_seconds = 2 ** (4 - retry_count)\n",
    "                print(f\"query={question}, error={e}, sleeping={sleeping_seconds}, remaining retry count={retry_count}\")\n",
    "                \n",
    "                time.sleep(sleeping_seconds)\n",
    "        \n",
    "        results.append(result)\n",
    "\n",
    "        time.sleep(1)\n",
    "    return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "fa6420f8-2d67-43bd-8ec6-2f136701e943",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-16T09:29:10.402987Z",
     "iopub.status.busy": "2024-12-16T09:29:10.402164Z",
     "iopub.status.idle": "2024-12-16T09:29:10.412210Z",
     "shell.execute_reply": "2024-12-16T09:29:10.409938Z",
     "shell.execute_reply.started": "2024-12-16T09:29:10.402864Z"
    }
   },
   "outputs": [],
   "source": [
    "eval_save_path = os.path.join(expr_dir, 'evals.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "ed65b04c-0b2a-49ae-9892-199f4ca8ee36",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-16T09:29:10.933399Z",
     "iopub.status.busy": "2024-12-16T09:29:10.932630Z",
     "iopub.status.idle": "2024-12-16T09:29:10.944208Z",
     "shell.execute_reply": "2024-12-16T09:29:10.941802Z",
     "shell.execute_reply.started": "2024-12-16T09:29:10.933332Z"
    }
   },
   "outputs": [],
   "source": [
    "if os.path.exists(eval_save_path):\n",
    "    print(f\"found cache at {eval_save_path}\")\n",
    "    eval_dict = pickle.load(open(eval_save_path, 'rb'))\n",
    "else:\n",
    "    eval_dict = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "d1e2d6c9-c45f-442b-8181-66c30437a308",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-16T09:29:11.943064Z",
     "iopub.status.busy": "2024-12-16T09:29:11.942242Z",
     "iopub.status.idle": "2024-12-16T09:34:14.346560Z",
     "shell.execute_reply": "2024-12-16T09:34:14.345077Z",
     "shell.execute_reply.started": "2024-12-16T09:29:11.942995Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "evaluating for pdf ...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1d76b66ce745419fab130338213ba1fc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "key=pdf raw_score unique: ['是' '否'], accuracy=0.53\n",
      "evaluating for markdown ...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9ff617aeb63341bb9efd2199594a56e7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "key=markdown raw_score unique: ['是' '否'], accuracy=0.69\n"
     ]
    }
   ],
   "source": [
    "metrics = []\n",
    "\n",
    "for key, vector_db in vector_db_dict.items():\n",
    "    print(f'evaluating for {key} ...')\n",
    "    if key in eval_dict:\n",
    "        pred_df = eval_dict[key]\n",
    "    else:\n",
    "        pred_df = pred_dict[key]\n",
    "        pred_df['raw_score'] = evaluate(pred_df)\n",
    "        eval_dict[key] = pred_df\n",
    "        pred_df['score'] = (pred_df['raw_score'] == '是').astype(int)\n",
    "        \n",
    "    print(f\"key={key} raw_score unique: {pred_df['raw_score'].unique()}, accuracy={pred_df['score'].mean()}\")\n",
    "    metrics.append({\n",
    "        'key': key,\n",
    "        'accuracy': pred_df['score'].mean()\n",
    "    })\n",
    "    pickle.dump(eval_dict, open(eval_save_path, 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "094615dd-3ce3-4b2b-a9b2-0f0044394891",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-16T09:34:14.349356Z",
     "iopub.status.busy": "2024-12-16T09:34:14.348810Z",
     "iopub.status.idle": "2024-12-16T09:34:14.358540Z",
     "shell.execute_reply": "2024-12-16T09:34:14.356072Z",
     "shell.execute_reply.started": "2024-12-16T09:34:14.349305Z"
    }
   },
   "outputs": [],
   "source": [
    "metrics_df = pd.DataFrame(metrics)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "c7702b66-c5da-4a59-95a1-83a377a3323a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-16T09:34:14.361638Z",
     "iopub.status.busy": "2024-12-16T09:34:14.360941Z",
     "iopub.status.idle": "2024-12-16T09:34:14.387161Z",
     "shell.execute_reply": "2024-12-16T09:34:14.384778Z",
     "shell.execute_reply.started": "2024-12-16T09:34:14.361572Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>key</th>\n",
       "      <th>accuracy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>pdf</td>\n",
       "      <td>0.53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>markdown</td>\n",
       "      <td>0.69</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        key  accuracy\n",
       "0       pdf      0.53\n",
       "1  markdown      0.69"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "b537c8af-bc72-40be-ba57-a613ebf531f1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-12-16T09:34:14.392062Z",
     "iopub.status.busy": "2024-12-16T09:34:14.391327Z",
     "iopub.status.idle": "2024-12-16T09:34:14.764912Z",
     "shell.execute_reply": "2024-12-16T09:34:14.764391Z",
     "shell.execute_reply.started": "2024-12-16T09:34:14.391994Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Axes: xlabel='key', ylabel='accuracy'>"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjcAAAGwCAYAAABVdURTAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/TGe4hAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAqUElEQVR4nO3df1RVdb7/8dcBBUQFNRSUSPyVP0rFIAhTYTUUTo5Xs6vkakK5yjSppXHHKUrx6jTSDzWcZKI0mmpMKacpm9RyKG1URhRH8ydOTgWlBzELFCcwzvn+0bfTPVcwOCIbPj4fa+21Oh/23ud9XEt5ts8+YHM6nU4BAAAYwsvqAQAAAJoScQMAAIxC3AAAAKMQNwAAwCjEDQAAMApxAwAAjELcAAAAo7SxeoDm5nA4dPz4cXXs2FE2m83qcQAAQAM4nU6dOXNGPXr0kJfXxa/NXHFxc/z4cYWFhVk9BgAA8EBpaamuvvrqi+5zxcVNx44dJX33hxMQEGDxNAAAoCEqKysVFhbm+j5+MVdc3Hz/VlRAQABxAwBAK9OQW0q4oRgAABiFuAEAAEYhbgAAgFGIGwAAYBTiBgAAGIW4AQAARmkRcZOdna3w8HD5+fkpJiZGhYWF9e4bHx8vm812wTZmzJhmnBgAALRUlsdNXl6e0tLStGDBAu3Zs0dDhw5VYmKiTp48Wef+b7zxhk6cOOHaDhw4IG9vb02cOLGZJwcAAC2R5XGzbNkypaamKiUlRYMGDVJOTo78/f2Vm5tb5/5dunRRSEiIa9u8ebP8/f2JGwAAIMniuKmpqVFRUZESEhJca15eXkpISFBBQUGDzvHCCy/orrvuUvv27ev8enV1tSorK902AABgLkvj5tSpU6qtrVVwcLDbenBwsOx2+48eX1hYqAMHDmj69On17pOZmanAwEDXxi/NBADAbJa/LXUpXnjhBQ0ePFjR0dH17pOenq6KigrXVlpa2owTAgCA5mbpL84MCgqSt7e3ysrK3NbLysoUEhJy0WOrqqq0du1aLVq06KL7+fr6ytfX95JnBQAArYOlV258fHwUGRmp/Px815rD4VB+fr5iY2Mveuzrr7+u6upq/fznP7/cYwIAgFbE0is3kpSWlqYpU6YoKipK0dHRysrKUlVVlVJSUiRJycnJCg0NVWZmpttxL7zwgsaPH6+rrrrKirEBAEALZXncJCUlqby8XBkZGbLb7YqIiNCmTZtcNxmXlJTIy8v9AlNxcbG2bdum9957z4qRAVzhShYNtnoEoMW5JmO/1SO42JxOp9PqIZpTZWWlAgMDVVFRoYCAAKvHAdAKETfAhS533DTm+3er/rQUAADA/0XcAAAAoxA3AADAKMQNAAAwCnEDAACMQtwAAACjEDcAAMAoxA0AADAKcQMAAIxC3AAAAKMQNwAAwCjEDQAAMApxAwAAjELcAAAAoxA3AADAKMQNAAAwCnEDAACMQtwAAACjEDcAAMAoxA0AADAKcQMAAIxC3AAAAKMQNwAAwCjEDQAAMApxAwAAjELcAAAAoxA3AADAKMQNAAAwCnEDAACMQtwAAACjEDcAAMAoxA0AADAKcQMAAIxC3AAAAKMQNwAAwCjEDQAAMApxAwAAjELcAAAAoxA3AADAKMQNAAAwiuVxk52drfDwcPn5+SkmJkaFhYUX3f/rr7/WzJkz1b17d/n6+uraa6/Vhg0bmmlaAADQ0rWx8snz8vKUlpamnJwcxcTEKCsrS4mJiSouLla3bt0u2L+mpka33nqrunXrpnXr1ik0NFSfffaZOnXq1PzDAwCAFsnSuFm2bJlSU1OVkpIiScrJydE777yj3NxcPfzwwxfsn5ubq9OnT2vHjh1q27atJCk8PLw5RwYAAC2cZW9L1dTUqKioSAkJCT8M4+WlhIQEFRQU1HnM+vXrFRsbq5kzZyo4OFjXX3+9Fi9erNra2nqfp7q6WpWVlW4bAAAwl2Vxc+rUKdXW1io4ONhtPTg4WHa7vc5j/vWvf2ndunWqra3Vhg0bNH/+fC1dulSPPfZYvc+TmZmpwMBA1xYWFtakrwMAALQslt9Q3BgOh0PdunXT888/r8jISCUlJenRRx9VTk5Ovcekp6eroqLCtZWWljbjxAAAoLlZds9NUFCQvL29VVZW5rZeVlamkJCQOo/p3r272rZtK29vb9fawIEDZbfbVVNTIx8fnwuO8fX1la+vb9MODwAAWizLrtz4+PgoMjJS+fn5rjWHw6H8/HzFxsbWeczNN9+sjz/+WA6Hw7V29OhRde/evc6wAQAAVx5L35ZKS0vTypUr9dJLL+nw4cO67777VFVV5fr0VHJystLT013733fffTp9+rRmz56to0eP6p133tHixYs1c+ZMq14CAABoYSz9KHhSUpLKy8uVkZEhu92uiIgIbdq0yXWTcUlJiby8fuivsLAwvfvuu3rwwQc1ZMgQhYaGavbs2XrooYesegkAAKCFsTmdTqfVQzSnyspKBQYGqqKiQgEBAVaPA6AVKlk02OoRgBbnmoz9l/X8jfn+3ao+LQUAAPBjiBsAAGAU4gYAABiFuAEAAEYhbgAAgFGIGwAAYBTiBgAAGIW4AQAARiFuAACAUYgbAABgFOIGAAAYhbgBAABGIW4AAIBRiBsAAGAU4gYAABiFuAEAAEYhbgAAgFGIGwAAYBTiBgAAGIW4AQAARiFuAACAUYgbAABgFOIGAAAYhbgBAABGIW4AAIBRiBsAAGAU4gYAABiFuAEAAEZpY/UApoqc+7LVIwAtTtFTyVaPAOAKwJUbAABgFOIGAAAYhbgBAABGIW4AAIBRiBsAAGAU4gYAABiFuAEAAEYhbgAAgFGIGwAAYBTiBgAAGIW4AQAARmkRcZOdna3w8HD5+fkpJiZGhYWF9e77hz/8QTabzW3z8/NrxmkBAEBLZnnc5OXlKS0tTQsWLNCePXs0dOhQJSYm6uTJk/UeExAQoBMnTri2zz77rBknBgAALZnlcbNs2TKlpqYqJSVFgwYNUk5Ojvz9/ZWbm1vvMTabTSEhIa4tODi4GScGAAAtmaVxU1NTo6KiIiUkJLjWvLy8lJCQoIKCgnqPO3v2rHr27KmwsDCNGzdOBw8erHff6upqVVZWum0AAMBclsbNqVOnVFtbe8GVl+DgYNnt9jqP6d+/v3Jzc/XWW2/pj3/8oxwOh4YPH67PP/+8zv0zMzMVGBjo2sLCwpr8dQAAgJbD8relGis2NlbJycmKiIhQXFyc3njjDXXt2lXPPfdcnfunp6eroqLCtZWWljbzxAAAoDm1sfLJg4KC5O3trbKyMrf1srIyhYSENOgcbdu21bBhw/Txxx/X+XVfX1/5+vpe8qwAAKB1sPTKjY+PjyIjI5Wfn+9aczgcys/PV2xsbIPOUVtbq/3796t79+6Xa0wAANCKWHrlRpLS0tI0ZcoURUVFKTo6WllZWaqqqlJKSookKTk5WaGhocrMzJQkLVq0SDfddJP69u2rr7/+Wk899ZQ+++wzTZ8+3cqXAQAAWgjL4yYpKUnl5eXKyMiQ3W5XRESENm3a5LrJuKSkRF5eP1xg+uqrr5Samiq73a7OnTsrMjJSO3bs0KBBg6x6CQAAoAWxOZ1Op9VDNKfKykoFBgaqoqJCAQEBl+15Iue+fNnODbRWRU8lWz1CkyhZNNjqEYAW55qM/Zf1/I35/t3qPi0FAABwMcQNAAAwCnEDAACMQtwAAACjEDcAAMAoxA0AADAKcQMAAIxC3AAAAKMQNwAAwCjEDQAAMApxAwAAjELcAAAAoxA3AADAKMQNAAAwCnEDAACMQtwAAACjEDcAAMAoxA0AADAKcQMAAIxC3AAAAKMQNwAAwCjEDQAAMApxAwAAjELcAAAAoxA3AADAKMQNAAAwCnEDAACMQtwAAACjEDcAAMAoxA0AADAKcQMAAIxC3AAAAKMQNwAAwCjEDQAAMIpHcfPBBx809RwAAABNwqO4GT16tPr06aPHHntMpaWlTT0TAACAxzyKmy+++EKzZs3SunXr1Lt3byUmJuq1115TTU1NU88HAADQKB7FTVBQkB588EHt3btXO3fu1LXXXqsZM2aoR48eeuCBB7Rv376mnhMAAKBBLvmG4htuuEHp6emaNWuWzp49q9zcXEVGRmrkyJE6ePBgU8wIAADQYB7Hzfnz57Vu3Trdfvvt6tmzp959912tWLFCZWVl+vjjj9WzZ09NnDixKWcFAAD4UR7Fzf3336/u3bvr3nvv1bXXXqt//OMfKigo0PTp09W+fXuFh4dryZIlOnLkSIPOl52drfDwcPn5+SkmJkaFhYUNOm7t2rWy2WwaP368Jy8DAAAYqI0nBx06dEjPPPOMJkyYIF9f3zr3CQoKatBHxvPy8pSWlqacnBzFxMQoKytLiYmJKi4uVrdu3eo97tNPP9WvfvUrjRw50pOXAAAADOXRlZv8/HxNnjy53rCRpDZt2iguLu5Hz7Vs2TKlpqYqJSVFgwYNUk5Ojvz9/ZWbm1vvMbW1tbr77ru1cOFC9e7d25OXAAAADOVR3GRmZtYZH7m5uXriiScafJ6amhoVFRUpISHhh4G8vJSQkKCCgoJ6j1u0aJG6deumadOm/ehzVFdXq7Ky0m0DAADm8ihunnvuOQ0YMOCC9euuu045OTkNPs+pU6dUW1ur4OBgt/Xg4GDZ7fY6j9m2bZteeOEFrVy5skHPkZmZqcDAQNcWFhbW4PkAAEDr41Hc2O12de/e/YL1rl276sSJE5c8VH3OnDmje+65RytXrlRQUFCDjklPT1dFRYVr4ycqAwBgNo9uKA4LC9P27dvVq1cvt/Xt27erR48eDT5PUFCQvL29VVZW5rZeVlamkJCQC/Y/duyYPv30U40dO9a15nA4JH13j09xcbH69Onjdoyvr+9F7w0CAABm8ShuUlNTNWfOHJ0/f1633HKLpO9uMv71r3+t//7v/27weXx8fBQZGan8/HzXx7kdDofy8/M1a9asC/YfMGCA9u/f77Y2b948nTlzRsuXL+ctJwAA4FnczJ07V19++aVmzJjh+n1Sfn5+euihh5Sent6oc6WlpWnKlCmKiopSdHS0srKyVFVVpZSUFElScnKyQkNDlZmZKT8/P11//fVux3fq1EmSLlgHAABXJo/ixmaz6YknntD8+fN1+PBhtWvXTv369fPo7Z+kpCSVl5crIyNDdrtdERER2rRpk+sm45KSEnl5XfJviQAAAFcIm9PpdFo9RHOqrKxUYGCgKioqFBAQcNmeJ3Luy5ft3EBrVfRUstUjNImSRYOtHgFoca7J2P/jO12Cxnz/9ujKjSTt3r1br732mkpKSlxvTX3vjTfe8PS0AAAAl8Sj93vWrl2r4cOH6/Dhw/rzn/+s8+fP6+DBg3r//fcVGBjY1DMCAAA0mEdxs3jxYj399NN6++235ePjo+XLl+vIkSOaNGmSrrnmmqaeEQAAoME8iptjx45pzJgxkr77OHdVVZVsNpsefPBBPf/88006IAAAQGN4FDedO3fWmTNnJEmhoaE6cOCAJOnrr7/WuXPnmm46AACARvLohuJRo0Zp8+bNGjx4sCZOnKjZs2fr/fff1+bNm/WTn/ykqWcEAABoMI/iZsWKFfrmm28kSY8++qjatm2rHTt26M4779S8efOadEAAAIDGaHTcfPvtt/rLX/6ixMRESZKXl5cefvjhJh8MAADAE42+56ZNmzb65S9/6bpyAwAA0JJ4dENxdHS09u7d28SjAAAAXDqP7rmZMWOG0tLSVFpaqsjISLVv397t60OGDGmS4QAAABrLo7i56667JEkPPPCAa81ms8npdMpms6m2trZppgMAAGgkj+Lmk08+aeo5AAAAmoRHcdOzZ8+mngMAAKBJeBQ3L7/88kW/npyc7NEwAAAAl8qjuJk9e7bb4/Pnz+vcuXPy8fGRv78/cQMAACzj0UfBv/rqK7ft7NmzKi4u1ogRI7RmzZqmnhEAAKDBPIqbuvTr10+PP/74BVd1AAAAmlOTxY303U8vPn78eFOeEgAAoFE8uudm/fr1bo+dTqdOnDihFStW6Oabb26SwQAAADzhUdyMHz/e7bHNZlPXrl11yy23aOnSpU0xFwAAgEc8ihuHw9HUcwAAADSJJr3nBgAAwGoexc2dd96pJ5544oL1J598UhMnTrzkoQAAADzlUdx8+OGHuv322y9Y/+lPf6oPP/zwkocCAADwlEdxc/bsWfn4+Fyw3rZtW1VWVl7yUAAAAJ7yKG4GDx6svLy8C9bXrl2rQYMGXfJQAAAAnvLo01Lz58/XhAkTdOzYMd1yyy2SpPz8fK1Zs0avv/56kw4IAADQGB7FzdixY/Xmm29q8eLFWrdundq1a6chQ4bor3/9q+Li4pp6RgAAgAbzKG4kacyYMRozZkxTzgIAAHDJPLrnZteuXdq5c+cF6zt37tTu3bsveSgAAABPeRQ3M2fOVGlp6QXrX3zxhWbOnHnJQwEAAHjKo7g5dOiQbrjhhgvWhw0bpkOHDl3yUAAAAJ7yKG58fX1VVlZ2wfqJEyfUpo3Ht/EAAABcMo/i5rbbblN6eroqKipca19//bUeeeQR3XrrrU02HAAAQGN5dJllyZIlGjVqlHr27Klhw4ZJkvbu3avg4GC98sorTTogAABAY3gUN6Ghofroo4+0evVq7du3T+3atVNKSoomT56stm3bNvWMAAAADebxDTLt27fXiBEjdM0116impkaStHHjRknSf/zHfzTNdAAAAI3kUdz861//0h133KH9+/fLZrPJ6XTKZrO5vl5bW9tkAwIAADSGRzcUz549W7169dLJkyfl7++vAwcOaOvWrYqKitKWLVuaeEQAAICG8yhuCgoKtGjRIgUFBcnLy0ve3t4aMWKEMjMz9cADDzT6fNnZ2QoPD5efn59iYmJUWFhY775vvPGGoqKi1KlTJ7Vv314RERHcxAwAAFw8ipva2lp17NhRkhQUFKTjx49Lknr27Kni4uJGnSsvL09paWlasGCB9uzZo6FDhyoxMVEnT56sc/8uXbro0UcfVUFBgT766COlpKQoJSVF7777ricvBQAAGMajuLn++uu1b98+SVJMTIyefPJJbd++XYsWLVLv3r0bda5ly5YpNTVVKSkpGjRokHJycuTv76/c3Nw694+Pj9cdd9yhgQMHqk+fPpo9e7aGDBmibdu2efJSAACAYTyKm3nz5snhcEiSFi1apE8++UQjR47Uhg0b9Lvf/a7B56mpqVFRUZESEhJ+GMjLSwkJCSooKPjR451Op/Lz81VcXKxRo0bVuU91dbUqKyvdNgAAYC6PPi2VmJjo+u++ffvqyJEjOn36tDp37uz2qakfc+rUKdXW1io4ONhtPTg4WEeOHKn3uIqKCoWGhqq6ulre3t76/e9/X+9PRs7MzNTChQsbPBMAAGjdPLpyU5cuXbo0KmwuRceOHbV3717t2rVLv/3tb5WWllbvp7S+/zUR3291/TZzAABgDkt/y2VQUJC8vb0v+CWcZWVlCgkJqfc4Ly8v9e3bV5IUERGhw4cPKzMzU/Hx8Rfs6+vrK19f3yadGwAAtFxNduXGEz4+PoqMjFR+fr5rzeFwKD8/X7GxsQ0+j8PhUHV19eUYEQAAtDKWXrmRpLS0NE2ZMkVRUVGKjo5WVlaWqqqqlJKSIklKTk5WaGioMjMzJX13D01UVJT69Omj6upqbdiwQa+88oqeffZZK18GAABoISyPm6SkJJWXlysjI0N2u10RERHatGmT6ybjkpISeXn9cIGpqqpKM2bM0Oeff6527dppwIAB+uMf/6ikpCSrXgIAAGhBbE6n02n1EM2psrJSgYGBqqioUEBAwGV7nsi5L1+2cwOtVdFTyVaP0CRKFg22egSgxbkmY/9lPX9jvn9bes8NAABAUyNuAACAUYgbAABgFOIGAAAYhbgBAABGIW4AAIBRiBsAAGAU4gYAABiFuAEAAEYhbgAAgFGIGwAAYBTiBgAAGIW4AQAARiFuAACAUYgbAABgFOIGAAAYhbgBAABGIW4AAIBRiBsAAGAU4gYAABiFuAEAAEYhbgAAgFGIGwAAYBTiBgAAGIW4AQAARiFuAACAUYgbAABgFOIGAAAYhbgBAABGIW4AAIBRiBsAAGAU4gYAABiFuAEAAEYhbgAAgFGIGwAAYBTiBgAAGIW4AQAARiFuAACAUYgbAABgFOIGAAAYpUXETXZ2tsLDw+Xn56eYmBgVFhbWu+/KlSs1cuRIde7cWZ07d1ZCQsJF9wcAAFcWy+MmLy9PaWlpWrBggfbs2aOhQ4cqMTFRJ0+erHP/LVu2aPLkyfrggw9UUFCgsLAw3Xbbbfriiy+aeXIAANASWR43y5YtU2pqqlJSUjRo0CDl5OTI399fubm5de6/evVqzZgxQxERERowYIBWrVolh8Oh/Pz8Zp4cAAC0RJbGTU1NjYqKipSQkOBa8/LyUkJCggoKChp0jnPnzun8+fPq0qVLnV+vrq5WZWWl2wYAAMxladycOnVKtbW1Cg4OdlsPDg6W3W5v0Dkeeugh9ejRwy2Q/rfMzEwFBga6trCwsEueGwAAtFyWvy11KR5//HGtXbtWf/7zn+Xn51fnPunp6aqoqHBtpaWlzTwlAABoTm2sfPKgoCB5e3urrKzMbb2srEwhISEXPXbJkiV6/PHH9de//lVDhgypdz9fX1/5+vo2ybwAAKDls/TKjY+PjyIjI91uBv7+5uDY2Nh6j3vyySf1m9/8Rps2bVJUVFRzjAoAAFoJS6/cSFJaWpqmTJmiqKgoRUdHKysrS1VVVUpJSZEkJScnKzQ0VJmZmZKkJ554QhkZGXr11VcVHh7uujenQ4cO6tChg2WvAwAAtAyWx01SUpLKy8uVkZEhu92uiIgIbdq0yXWTcUlJiby8frjA9Oyzz6qmpkb/+Z//6XaeBQsW6H/+53+ac3QAANACWR43kjRr1izNmjWrzq9t2bLF7fGnn356+QcCAACtVqv+tBQAAMD/RdwAAACjEDcAAMAoxA0AADAKcQMAAIxC3AAAAKMQNwAAwCjEDQAAMApxAwAAjELcAAAAoxA3AADAKMQNAAAwCnEDAACMQtwAAACjEDcAAMAoxA0AADAKcQMAAIxC3AAAAKMQNwAAwCjEDQAAMApxAwAAjELcAAAAoxA3AADAKMQNAAAwCnEDAACMQtwAAACjEDcAAMAoxA0AADAKcQMAAIxC3AAAAKMQNwAAwCjEDQAAMApxAwAAjELcAAAAoxA3AADAKMQNAAAwCnEDAACMQtwAAACjEDcAAMAoxA0AADCK5XGTnZ2t8PBw+fn5KSYmRoWFhfXue/DgQd15550KDw+XzWZTVlZW8w0KAABaBUvjJi8vT2lpaVqwYIH27NmjoUOHKjExUSdPnqxz/3Pnzql37956/PHHFRIS0szTAgCA1sDSuFm2bJlSU1OVkpKiQYMGKScnR/7+/srNza1z/xtvvFFPPfWU7rrrLvn6+jbztAAAoDWwLG5qampUVFSkhISEH4bx8lJCQoIKCgqa7Hmqq6tVWVnptgEAAHNZFjenTp1SbW2tgoOD3daDg4Nlt9ub7HkyMzMVGBjo2sLCwprs3AAAoOWx/Ibiyy09PV0VFRWurbS01OqRAADAZdTGqicOCgqSt7e3ysrK3NbLysqa9GZhX19f7s8BAOAKYtmVGx8fH0VGRio/P9+15nA4lJ+fr9jYWKvGAgAArZxlV24kKS0tTVOmTFFUVJSio6OVlZWlqqoqpaSkSJKSk5MVGhqqzMxMSd/dhHzo0CHXf3/xxRfau3evOnTooL59+1r2OgAAQMthadwkJSWpvLxcGRkZstvtioiI0KZNm1w3GZeUlMjL64eLS8ePH9ewYcNcj5csWaIlS5YoLi5OW7Zsae7xAQBAC2Rp3EjSrFmzNGvWrDq/9n+DJTw8XE6nsxmmAgAArZXxn5YCAABXFuIGAAAYhbgBAABGIW4AAIBRiBsAAGAU4gYAABiFuAEAAEYhbgAAgFGIGwAAYBTiBgAAGIW4AQAARiFuAACAUYgbAABgFOIGAAAYhbgBAABGIW4AAIBRiBsAAGAU4gYAABiFuAEAAEYhbgAAgFGIGwAAYBTiBgAAGIW4AQAARiFuAACAUYgbAABgFOIGAAAYhbgBAABGIW4AAIBRiBsAAGAU4gYAABiFuAEAAEYhbgAAgFGIGwAAYBTiBgAAGIW4AQAARiFuAACAUYgbAABgFOIGAAAYhbgBAABGIW4AAIBRWkTcZGdnKzw8XH5+foqJiVFhYeFF93/99dc1YMAA+fn5afDgwdqwYUMzTQoAAFo6y+MmLy9PaWlpWrBggfbs2aOhQ4cqMTFRJ0+erHP/HTt2aPLkyZo2bZr+8Y9/aPz48Ro/frwOHDjQzJMDAICWyPK4WbZsmVJTU5WSkqJBgwYpJydH/v7+ys3NrXP/5cuXa/To0Zo7d64GDhyo3/zmN7rhhhu0YsWKZp4cAAC0RG2sfPKamhoVFRUpPT3dtebl5aWEhAQVFBTUeUxBQYHS0tLc1hITE/Xmm2/WuX91dbWqq6tdjysqKiRJlZWVlzj9xdVW//uynh9ojS7337vmcuabWqtHAFqcy/33+/vzO53OH93X0rg5deqUamtrFRwc7LYeHBysI0eO1HmM3W6vc3+73V7n/pmZmVq4cOEF62FhYR5ODcBTgc/80uoRAFwumYHN8jRnzpxRYODFn8vSuGkO6enpbld6HA6HTp8+rauuuko2m83CydAcKisrFRYWptLSUgUEBFg9DoAmxN/vK4vT6dSZM2fUo0ePH93X0rgJCgqSt7e3ysrK3NbLysoUEhJS5zEhISGN2t/X11e+vr5ua506dfJ8aLRKAQEB/OMHGIq/31eOH7ti8z1Lbyj28fFRZGSk8vPzXWsOh0P5+fmKjY2t85jY2Fi3/SVp8+bN9e4PAACuLJa/LZWWlqYpU6YoKipK0dHRysrKUlVVlVJSUiRJycnJCg0NVWZmpiRp9uzZiouL09KlSzVmzBitXbtWu3fv1vPPP2/lywAAAC2E5XGTlJSk8vJyZWRkyG63KyIiQps2bXLdNFxSUiIvrx8uMA0fPlyvvvqq5s2bp0ceeUT9+vXTm2++qeuvv96ql4AWzNfXVwsWLLjgrUkArR9/v1Efm7Mhn6kCAABoJSz/IX4AAABNibgBAABGIW4AAIBRiBtcUcLDw5WVleV6bLfbdeutt6p9+/b8/COgBfq/f2cbIj4+XnPmzLks86B1sPzTUoCVnn76aZ04cUJ79+5t8A+HAgC0bMQNrmjHjh1TZGSk+vXrZ/UoAP6Xmpoa+fj4WD0GWinelkKrFR8fr1mzZmnWrFkKDAxUUFCQ5s+f7/qNsSdPntTYsWPVrl079erVS6tXr3Y7Pjw8XH/605/08ssvy2azaerUqRa8CqD1iY+P1/333685c+aoc+fOCg4O1sqVK10/gLVjx47q27evNm7cKEmqra3VtGnT1KtXL7Vr1079+/fX8uXL3c45depUjR8/Xr/97W/Vo0cP9e/fv87nXrVqlTp16uT6SfVVVVVKTk5Whw4d1L17dy1duvSCY7766islJyerc+fO8vf3109/+lP985//lPTd7yvq2rWr1q1b59o/IiJC3bt3dz3etm2bfH19de7cOUmSzWbTqlWrdMcdd8jf31/9+vXT+vXrL+FPFE2NuEGr9tJLL6lNmzYqLCzU8uXLtWzZMq1atUrSd/9YlpaW6oMPPtC6dev0+9//XidPnnQdu2vXLo0ePVqTJk3SiRMnLvjHFkD9XnrpJQUFBamwsFD333+/7rvvPk2cOFHDhw/Xnj17dNttt+mee+7RuXPn5HA4dPXVV+v111/XoUOHlJGRoUceeUSvvfaa2znz8/NVXFyszZs36y9/+csFz/nkk0/q4Ycf1nvvvaef/OQnkqS5c+dq69ateuutt/Tee+9py5Yt2rNnj9txU6dO1e7du7V+/XoVFBTI6XTq9ttv1/nz52Wz2TRq1Cht2bJF0nchdPjwYf373//WkSNHJElbt27VjTfeKH9/f9c5Fy5cqEmTJumjjz7S7bffrrvvvlunT59uyj9iXAon0ErFxcU5Bw4c6HQ4HK61hx56yDlw4EBncXGxU5KzsLDQ9bXDhw87JTmffvpp19q4ceOcU6ZMacapgdYvLi7OOWLECNfjb7/91tm+fXvnPffc41o7ceKEU5KzoKCgznPMnDnTeeedd7oeT5kyxRkcHOysrq52269nz57Op59+2vnrX//a2b17d+eBAwdcXztz5ozTx8fH+dprr7nWvvzyS2e7du2cs2fPdjqdTufRo0edkpzbt2937XPq1Clnu3btXMf97ne/c1533XVOp9PpfPPNN50xMTHOcePGOZ999lmn0+l0JiQkOB955BHX8ZKc8+bNcz0+e/asU5Jz48aNP/Inh+bClRu0ajfddJNsNpvrcWxsrP75z3/q8OHDatOmjSIjI11fGzBgAJ+IAprIkCFDXP/t7e2tq666SoMHD3atff8rdL6/Wpqdna3IyEh17dpVHTp00PPPP6+SkhK3cw4ePLjO+2yWLl2qlStXatu2bbruuutc68eOHVNNTY1iYmJca126dHF7S+v7fwv+9z5XXXWV+vfvr8OHD0uS4uLidOjQIZWXl2vr1q2Kj49XfHy8tmzZovPnz2vHjh2Kj4+v9/W3b99eAQEBbleGYS3iBgDQaG3btnV7bLPZ3Na+/58Oh8OhtWvX6le/+pWmTZum9957T3v37lVKSopqamrcztG+ffs6n2vkyJGqra294G2spjJ48GB16dJFW7dudYubrVu3ateuXTp//ryGDx/udkxdr9/hcFyW+dB4xA1atZ07d7o9/vvf/65+/fppwIAB+vbbb1VUVOT6WnFxsb7++utmnhDA9u3bNXz4cM2YMUPDhg1T3759dezYsQYfHx0drY0bN2rx4sVasmSJa71Pnz5q27at278DX331lY4ePep6PHDgQH377bdu+3z55ZcqLi7WoEGDJH0XJiNHjtRbb72lgwcPasSIERoyZIiqq6v13HPPKSoqqt7wQstE3KBVKykpUVpamoqLi7VmzRo988wzmj17tvr376/Ro0fr3nvv1c6dO1VUVKTp06erXbt2Vo8MXHH69eun3bt3691339XRo0c1f/587dq1q1HnGD58uDZs2KCFCxe6fqhfhw4dNG3aNM2dO1fvv/++Dhw4oKlTp8rL64dvbf369dO4ceOUmpqqbdu2ad++ffr5z3+u0NBQjRs3zrVffHy81qxZo4iICHXo0EFeXl4aNWqUVq9erbi4uCb5c0DzIW7QqiUnJ+vf//63oqOjNXPmTM2ePVu/+MUvJEkvvviievToobi4OE2YMEG/+MUv1K1bN4snBq489957ryZMmKCkpCTFxMToyy+/1IwZMxp9nhEjRuidd97RvHnz9Mwzz0iSnnrqKY0cOVJjx45VQkKCRowY4XavnfTdvwWRkZH62c9+ptjYWDmdTm3YsMHtraW4uDjV1ta63VsTHx9/wRpaB5vT+f9/KAjQysTHxysiIqLRP5odAGA2rtwAAACjEDcAAMAovC0FAACMwpUbAABgFOIGAAAYhbgBAABGIW4AAIBRiBsAAGAU4gZAixcfH685c+ZYPQaAVoK4AQAARiFuAACAUYgbAK3OO++8o8DAQK1evVqlpaWaNGmSOnXqpC5dumjcuHH69NNPJUkffvih2rZtK7vd7nb8nDlzNHLkSAsmB9AciBsArcqrr76qyZMna/Xq1Zo0aZISExPVsWNH/e1vf9P27dvVoUMHjR49WjU1NRo1apR69+6tV155xXX8+fPntXr1av3Xf/2Xha8CwOVE3ABoNbKzszVjxgy9/fbb+tnPfqa8vDw5HA6tWrVKgwcP1sCBA/Xiiy+qpKREW7ZskSRNmzZNL774ouscb7/9tr755htNmjTJolcB4HJrY/UAANAQ69at08mTJ7V9+3bdeOONkqR9+/bp448/VseOHd32/eabb3Ts2DFJ0tSpUzVv3jz9/e9/10033aQ//OEPmjRpktq3b9/srwFA8yBuALQKw4YN0549e5Sbm6uoqCjZbDadPXtWkZGRWr169QX7d+3aVZLUrVs3jR07Vi+++KJ69eqljRs3uq7qADATcQOgVejTp4+WLl2q+Ph4eXt7a8WKFbrhhhuUl5enbt26KSAgoN5jp0+frsmTJ+vqq69Wnz59dPPNNzfj5ACaG/fcAGg1rr32Wn3wwQf605/+pDlz5ujuu+9WUFCQxo0bp7/97W/65JNPtGXLFj3wwAP6/PPPXcclJiYqICBAjz32mFJSUix8BQCaA3EDoFXp37+/3n//fa1Zs0bz58/Xhx9+qGuuuUYTJkzQwIEDNW3aNH3zzTduV3K8vLw0depU1dbWKjk52cLpATQHm9PpdFo9BABcbtOmTVN5ebnWr19v9SgALjPuuQFgtIqKCu3fv1+vvvoqYQNcIYgbAEYbN26cCgsL9ctf/lK33nqr1eMAaAa8LQUAAIzCDcUAAMAoxA0AADAKcQMAAIxC3AAAAKMQNwAAwCjEDQAAMApxAwAAjELcAAAAo/w/eb6bzrCKZHMAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import seaborn as sns\n",
    "\n",
    "sns.barplot(x='key', y='accuracy', data=metrics_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9fc0c11c-be15-47dc-88bf-31b0192b4622",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 1058.563616,
   "end_time": "2024-11-23T14:46:37.625874",
   "environment_variables": {},
   "exception": null,
   "input_path": "13_contextual_embeddings.ipynb",
   "output_path": "run_13_contextual_embeddings.ipynb",
   "parameters": {},
   "start_time": "2024-11-23T14:28:59.062258",
   "version": "2.6.0"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {
     "0cd8c168767249f2a5fa412173f6e751": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_5ce1d1d9d86c40d9839877ff95734491",
       "max": 100,
       "min": 0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_231702cf4d79477f9d5548665a1b18fe",
       "tabbable": null,
       "tooltip": null,
       "value": 100
      }
     },
     "2133bb8d85d34b8db112b4408ad60320": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "background": null,
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "231702cf4d79477f9d5548665a1b18fe": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "23b1ad9c0f9c46c888da66e85c90eb84": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "24e6eadc3dc940ecabf30dd1a3c6d1f3": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_fa4bddf2c33241b5bf918054518f128f",
       "max": 52,
       "min": 0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_edc33e82be8f41eba6a18a0ef074ab7a",
       "tabbable": null,
       "tooltip": null,
       "value": 52
      }
     },
     "2f60367b1c8941e2bf71661c33969ae8": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "3865f25c78aa46f29a25d807205281c3": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "3d0b06deaa654b989eece8cde06fa0f8": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "background": null,
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "3f8ceda83287475b97608e42f5f6782f": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "4881e496f1c84fe29ce9ebebaddfb3c2": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_bd096d5d219a467786a85cfe1613fedd",
        "IPY_MODEL_24e6eadc3dc940ecabf30dd1a3c6d1f3",
        "IPY_MODEL_bc2b8104b4244d8cacedeb95e800d91c"
       ],
       "layout": "IPY_MODEL_6b9a8e43c1c342dba500a14e7149b600",
       "tabbable": null,
       "tooltip": null
      }
     },
     "5ce1d1d9d86c40d9839877ff95734491": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "5ddb08be5cc64c9ab40a1d62a21763a5": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_86283159049d48b1adcfb2de2d404d4d",
       "placeholder": "​",
       "style": "IPY_MODEL_2133bb8d85d34b8db112b4408ad60320",
       "tabbable": null,
       "tooltip": null,
       "value": " 100/100 [08:34&lt;00:00, 10.01s/it]"
      }
     },
     "5ef9d83ccad1471f85335900a24a8553": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "6b9a8e43c1c342dba500a14e7149b600": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "816a079a8c804fbfa9b9a74f941abea8": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_bcc69ec5db1b4aab977807284c9290e7",
        "IPY_MODEL_0cd8c168767249f2a5fa412173f6e751",
        "IPY_MODEL_5ddb08be5cc64c9ab40a1d62a21763a5"
       ],
       "layout": "IPY_MODEL_d1178c6858284f788a80b5f2a14fd0b7",
       "tabbable": null,
       "tooltip": null
      }
     },
     "86283159049d48b1adcfb2de2d404d4d": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "8ff8262c56604119883f4a5f13bb74ab": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_5ef9d83ccad1471f85335900a24a8553",
       "placeholder": "​",
       "style": "IPY_MODEL_e89e77133c344fc48c1d62f5a607ec93",
       "tabbable": null,
       "tooltip": null,
       "value": " 8/8 [00:18&lt;00:00,  2.27s/it]"
      }
     },
     "9189a076554543aaa6f5ee04e40dbe1b": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "background": null,
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "988e6697a2af486fadeaf0b84347b565": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_e1aae4c55cb64f379e74f15357275628",
        "IPY_MODEL_fd9e23198ca1489a9773fda3510bf857",
        "IPY_MODEL_8ff8262c56604119883f4a5f13bb74ab"
       ],
       "layout": "IPY_MODEL_d2ee15001d2244529f7e47d3333c0f8e",
       "tabbable": null,
       "tooltip": null
      }
     },
     "9fc7d91f94a94933bde5ba80e64587de": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "background": null,
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "a7d240a289084bdfba4724c0efd5ab07": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "bc2b8104b4244d8cacedeb95e800d91c": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_2f60367b1c8941e2bf71661c33969ae8",
       "placeholder": "​",
       "style": "IPY_MODEL_9fc7d91f94a94933bde5ba80e64587de",
       "tabbable": null,
       "tooltip": null,
       "value": " 52/52 [04:26&lt;00:00,  4.22s/it]"
      }
     },
     "bcc69ec5db1b4aab977807284c9290e7": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_3f8ceda83287475b97608e42f5f6782f",
       "placeholder": "​",
       "style": "IPY_MODEL_3d0b06deaa654b989eece8cde06fa0f8",
       "tabbable": null,
       "tooltip": null,
       "value": "100%"
      }
     },
     "bd096d5d219a467786a85cfe1613fedd": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_3865f25c78aa46f29a25d807205281c3",
       "placeholder": "​",
       "style": "IPY_MODEL_9189a076554543aaa6f5ee04e40dbe1b",
       "tabbable": null,
       "tooltip": null,
       "value": "100%"
      }
     },
     "cc3ed8dc4a5c43aca7b62d904865b2fa": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "cf68b6fe24964ce792aa63827489cb97": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "background": null,
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "d1178c6858284f788a80b5f2a14fd0b7": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "d2ee15001d2244529f7e47d3333c0f8e": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "e1aae4c55cb64f379e74f15357275628": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_23b1ad9c0f9c46c888da66e85c90eb84",
       "placeholder": "​",
       "style": "IPY_MODEL_cf68b6fe24964ce792aa63827489cb97",
       "tabbable": null,
       "tooltip": null,
       "value": "100%"
      }
     },
     "e89e77133c344fc48c1d62f5a607ec93": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "background": null,
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "edc33e82be8f41eba6a18a0ef074ab7a": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "fa4bddf2c33241b5bf918054518f128f": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "fd9e23198ca1489a9773fda3510bf857": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_cc3ed8dc4a5c43aca7b62d904865b2fa",
       "max": 8,
       "min": 0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_a7d240a289084bdfba4724c0efd5ab07",
       "tabbable": null,
       "tooltip": null,
       "value": 8
      }
     }
    },
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
